# =============================================================================
# Script to download shapefiles from TIGER (US Govt. Database of Geographic Shapefiles)
# =============================================================================

# =============================================================================
#Choose the Geography of interest and the Year (vintage) of the map
geos=['BG','STATE','ZCTA5','TRACT']
y=2019
#Other geographies include:
#TABBLOCK
#CD is congressional districts, 
#ELSD is school districts (subset of states),
#COUSUB is county subparts, state legislative districts (upper and lower are SLDU SLDL)
for geo in geos:
    loc = 'geo/tiger/TIGER'+str(y)+'/'+geo
    # define the name of the directory to be created
    out_loc = cdd['p_d_geo_shp_tiger'] + '/' + geo + '_' + str(y)
    ftp = FTP('ftp2.census.gov')
    ftp.login(user='anonymous',passwd='anonymous')
    
    try:
        os.makedirs(out_loc)
    except OSError:
        pass
    # Get All Files
    ftp.cwd(loc)
    files = ftp.nlst()
    # Print out the files
    for file in tqdm(files):
        outfile = out_loc + r'\\' + file
        #Check if the outfile exists
        if os.path.exists(outfile):
            print(outfile + ' already downloaded')
            continue
        for i in range(10):
            try:
                ftp.retrbinary("RETR " + file ,open(outfile, 'wb').write)
                break
            except:
                ftp.close()
                ftp = FTP('ftp2.census.gov')
                ftp.login(user='anonymous',passwd='anonymous')
                pass
    ftp.close()


#Download the 2010 Census Tract and Block Group shapes. 
#   For BG, TRACT, and TABBLOCK, the TIGER folder contains both state and tract level aggregation zip files. 
#   By only choosing files with short names I am downloading the fewer state level files.
y1=2010
y2=2010
geos=['TABBLOCK']
for geo in geos:
    loc = 'geo/tiger/TIGER'+str(y1)+'/'+geo+'/'+str(y2)
    # define the name of the directory to be created
    out_loc = cdd['p_d_geo_shp_tiger'] + '/' + geo + '_' + str(y2)
    ftp = FTP('ftp2.census.gov')
    ftp.login(user='anonymous',passwd='anonymous')
    
    try:
        os.makedirs(out_loc)
    except OSError:
        pass
    # Get All Files
    ftp.cwd(loc)
    files = ftp.nlst()
    if geo=='BG':
        files = [f for f in files if len(f)<21]
    if geo=='TRACT':
        files = [f for f in files if len(f)<25]
    if geo=='TABBLOCK':
        files = [f for f in files if len(f)<28] #Note, the tabblock files are large so changing this to len(f)==28 will download the data by county which may be less error prone.
    
    # Print out the files
    for file in tqdm(files):
        outfile = out_loc + r'\\' + file
        #Check if the outfile exists
        if os.path.exists(outfile):
            print(outfile + ' already downloaded')
            continue
        for i in range(10):
            try:
                ftp.retrbinary("RETR " + file ,open(outfile, 'wb').write)
                break
            except:
                ftp.close()
                ftp = FTP('ftp2.census.gov')
                ftp.login(user='anonymous',passwd='anonymous')
                pass
    ftp.close()



# =============================================================================
# These are tiger files with demographics
try:
    os.makedirs(cdd['p_d_geo_demo'])
except:
    pass
ftp = FTP('ftp2.census.gov')
ftp.login(user='anonymous',passwd='anonymous')
# Get All Block Level Demo Files
ftp.cwd('geo/tiger/TIGER2010BLKPOPHU/')
files = ftp.nlst()
# Print out the files
for file in tqdm(files):
    outfile = cdd['p_d_geo_demo'] + r'\\' + file
    #Check if the outfile exists
    if os.path.exists(outfile):
        print(outfile + ' already downloaded')
        continue
    for i in range(10):
        try:
            ftp.retrbinary("RETR " + file ,open(outfile, 'wb').write)
            break
        except:
            ftp.close()
            ftp = FTP('ftp2.census.gov')
            ftp.login(user='anonymous',passwd='anonymous')
            pass






